import scrapy
import re
import datetime
from ..items import Job51Item


class JobSpiderSpider(scrapy.Spider):
    name = 'job_spider'
    # allowed_domains = ['51job.com']
    start_urls = [
        # 关键字:数据分析
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590,2,',
        # 关键字:数据挖掘
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E6%258C%2596%25E6%258E%2598,2,',
        # 关键字:算法
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E7%25AE%2597%25E6%25B3%2595,2,',
        # 关键字:机器学习
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%259C%25BA%25E5%2599%25A8%25E5%25AD%25A6%25E4%25B9%25A0,2,',
        # 关键字:深度学习
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%25B7%25B1%25E5%25BA%25A6%25E5%25AD%25A6%25E4%25B9%25A0,2,',
        # 关键字:人工智能
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E4%25BA%25BA%25E5%25B7%25A5%25E6%2599%25BA%25E8%2583%25BD,2,'
    ]

    _url = '.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
    start_urls_tags = [
        '数据分析',
        '数据挖掘',
        '算法',
        '机器学习',
        '深度学习',
        '人工智能'
    ]

    # 爬虫初始化的时候获取爬取时间
    def __init__(self):
        self.record_date = datetime.datetime.now().strftime("%Y-%m-%d")

    # 遍历关键字网页,并实现翻页
    def start_requests(self):
        for ind, start_url in enumerate(self.start_urls):
            tag = self.start_urls_tags[ind]
            for page in range(1, 2001):
                url = start_url + str(page) + self._url
                # print(url)
                yield scrapy.Request(url, callback=self.parse, meta={'start_url': tag})

    def parse(self, response):
        # 获取javascript文本
        javascript_text = response.xpath('/html/body/script[2]/text()').extract_first()
        # print(javascript_text)

        # 获取详情页url
        detail_url = re.findall(r'"job_href":"(.+?)","job_name"', string=javascript_text)
        detail_urls = ','.join(detail_url).replace('\\', '').split(',')
        # print(detail_url)
        # print(detail_urls)
        # print(len(detail_urls))
        tag = response.meta['start_url']

        # 遍历详情页
        for ind, detail_page in enumerate(detail_urls):
            page = detail_urls[ind]
            yield scrapy.Request(detail_page, callback=self.parse_details, meta={'start_url': tag})

    def parse_details(self, response):
        # 招聘名称
        title = response.xpath('//div[@class="cn"]/h1/text()').extract()
        # print(title)
        # 职位信息
        position = response.xpath('//div[@class="bmsg job_msg inbox"]/p/text()').extract()
        # print(position)
        # 薪资
        salary = response.xpath('//div[@class="cn"]/strong/text()').extract()
        # print(salary)
        # 职位福利
        welfare = response.xpath('//div[@class="t1"]/span/text()').extract()
        # print(welfare)
        # 经验要求
        experience = ','.join(response.xpath('//p[@class="msg ltype"]/text()[2]').extract()).split()
        # print(experience)
        # 学历要求
        education = ','.join(response.xpath('//p[@class="msg ltype"]/text()[3]').extract()).split()
        # print(education)
        # 招聘人数
        hiring_num = ','.join(response.xpath('//p[@class="msg ltype"]/text()[4]').extract()).split()
        # print(hiring_num)
        # 发布时间
        time = ','.join(response.xpath('//p[@class="msg ltype"]/text()[5]').extract()).split()
        # print(time)
        # 公司名称
        company = response.xpath('//div[@class="com_msg"]/a/p/text()').extract()
        # print(company)
        # 公司性质
        attribute = response.xpath('//div[@class="com_tag"]/p[1]/text()').extract()
        # print(attribute)
        # 公司人数
        members = response.xpath('//div[@class="com_tag"]/p[2]/text()').extract()
        # print(members)
        # 公司行业
        industry = response.xpath('//div[@class="com_tag"]/p[3]/a/text()').extract()
        # print(industry)
        # 公司地址
        address = ','.join(response.xpath('//p[@class="msg ltype"]/text()[1]').extract()).split()
        # print(address)
        # 公司概况
        general_situation = response.xpath('//div[@class="tmsg inbox"]/text()').extract()
        # print(general_situation)
        tag = response.meta['start_url']

        # item = Job51Item()
        # item['record_date'] = self.record_date
        # item['tag']= tag
        #
        # item['title'] = title
        # item['position'] = position
        # item['salary'] = salary
        # item['welfare'] = welfare
        # item['experience'] = experience
        # item['education'] = education
        # item['hiring_num'] = hiring_num
        # item['time'] = time
        # item['company'] = company
        # item['attribute'] = attribute
        # item['members'] = members
        # item['industry'] = industry
        # item['address'] = address
        # item['general_situation'] = general_situation
        #
        # yield item
